{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/liyaze/anaconda3/envs/longchain/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from fastchat.serve.model_worker import app, GptqConfig, AWQConfig, ModelWorker, worker_id\n",
    "import sys\n",
    "from server.utils import (fschat_controller_address, fschat_model_worker_address,\n",
    "                          fschat_openai_api_address, get_httpx_client, get_model_worker_config,\n",
    "                          MakeFastAPIOffline, FastAPI, llm_device, embedding_device)\n",
    "from langchain_core._api import deprecated\n",
    "from configs import (\n",
    "    LOG_PATH,\n",
    "    log_verbose,\n",
    "    logger,\n",
    "    LLM_MODELS,\n",
    "    EMBEDDING_MODEL,\n",
    "    TEXT_SPLITTER_NAME,\n",
    "    FSCHAT_CONTROLLER,\n",
    "    FSCHAT_OPENAI_API,\n",
    "    FSCHAT_MODEL_WORKERS,\n",
    "    API_SERVER,\n",
    "    WEBUI_SERVER,\n",
    "    HTTPX_DEFAULT_TIMEOUT,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Namespace(device='auto', infer_turbo=False, model_names=['Qwen/Qwen1.5-72B-Chat'], controller_address='http://127.0.0.1:20001', worker_address='http://0.0.0.0:20002', model_path='Qwen/Qwen1.5-72B-Chat', gpus='0,1,2,3,4', max_gpu_memory='22GiB', num_gpus=5, load_8bit=False, cpu_offloading=None, gptq_ckpt=None, gptq_wbits=16, gptq_groupsize=-1, gptq_act_order=False, awq_ckpt=None, awq_wbits=16, awq_groupsize=-1, conv_template=None, limit_worker_concurrency=5, stream_interval=2, no_register=False, embed_in_truncate=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "@deprecated(\n",
    "    since=\"0.3.0\",\n",
    "    message=\"模型启动功能将于 Langchain-Chatchat 0.3.x重写,支持更多模式和加速启动，0.2.x中相关功能将废弃\",\n",
    "    removal=\"0.3.0\")\n",
    "def create_controller_app(\n",
    "        dispatch_method: str,\n",
    "        log_level: str = \"INFO\",\n",
    ") -> FastAPI:\n",
    "    import fastchat.constants\n",
    "    fastchat.constants.LOGDIR = LOG_PATH\n",
    "    from fastchat.serve.controller import app, Controller, logger\n",
    "    logger.setLevel(log_level)\n",
    "\n",
    "    controller = Controller(dispatch_method)\n",
    "    sys.modules[\"fastchat.serve.controller\"].controller = controller\n",
    "\n",
    "    MakeFastAPIOffline(app)\n",
    "    app.title = \"FastChat Controller\"\n",
    "    app._controller = controller\n",
    "    return app"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "device='auto'\n",
    "infer_turbo=False\n",
    "model_names=['01-ai/Yi-6B-Chat'] # ['Qwen/Qwen1.5-72B-Chat']\n",
    "controller_address='http://127.0.0.1:30001'\n",
    "worker_address='http://0.0.0.0:30002'\n",
    "model_path='Qwen/Qwen1.5-72B-Chat'\n",
    "gpus='1'\n",
    "max_gpu_memory='22GiB'\n",
    "num_gpus=1\n",
    "load_8bit=False\n",
    "cpu_offloading=None\n",
    "gptq_ckpt=None\n",
    "gptq_wbits=16\n",
    "gptq_groupsize=-1\n",
    "gptq_act_order=False\n",
    "awq_ckpt=None\n",
    "awq_wbits=16\n",
    "awq_groupsize=-1\n",
    "conv_template=None\n",
    "limit_worker_concurrency=5\n",
    "stream_interval=2\n",
    "no_register=False\n",
    "embed_in_truncate=False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "gptq_config = GptqConfig(\n",
    "    ckpt=gptq_ckpt or model_path,\n",
    "    wbits=gptq_wbits,\n",
    "    groupsize=gptq_groupsize,\n",
    "    act_order=gptq_act_order,\n",
    ")\n",
    "awq_config = AWQConfig(\n",
    "    ckpt=awq_ckpt or model_path,\n",
    "    wbits=awq_wbits,\n",
    "    groupsize=awq_groupsize,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-03-17 10:35:09 | ERROR | stderr | /home/liyaze/anaconda3/envs/longchain/lib/python3.11/site-packages/langchain_core/_api/deprecation.py:117: LangChainDeprecationWarning: 模型启动功能将于 Langchain-Chatchat 0.3.x重写,支持更多模式和加速启动，0.2.x中相关功能将废弃\n",
      "2024-03-17 10:35:09 | ERROR | stderr |   warn_deprecated(\n",
      "2024-03-17 10:35:34 | INFO | model_worker | Loading the model ['01-ai/Yi-6B-Chat'] on worker 01647377 ...\n",
      "2024-03-17 10:35:34 | INFO | stdout | \u001b[1;31m/home/liyaze/anaconda3/envs/longchain/lib/python3.11/site-packages/fastchat/serve/model_worker.py:78 : __init__\u001b[0m model_path: Qwen/Qwen1.5-72B-Chat\n",
      "2024-03-17 10:35:34 | INFO | stdout | \u001b[1;31m/home/liyaze/anaconda3/envs/longchain/lib/python3.11/site-packages/fastchat/model/model_adapter.py:196 : load_model\u001b[0m model_path: Qwen/Qwen1.5-72B-Chat\n",
      "Loading checkpoint shards:   0%|          | 0/38 [00:00<?, ?it/s]\n",
      "Loading checkpoint shards:   3%|▎         | 1/38 [00:09<05:45,  9.33s/it]\n",
      "Loading checkpoint shards:   5%|▌         | 2/38 [00:18<05:28,  9.13s/it]\n",
      "Loading checkpoint shards:   8%|▊         | 3/38 [00:22<04:00,  6.86s/it]\n",
      "Loading checkpoint shards:  11%|█         | 4/38 [00:23<02:28,  4.36s/it]\n",
      "Loading checkpoint shards:  13%|█▎        | 5/38 [00:23<01:37,  2.94s/it]\n",
      "Loading checkpoint shards:  16%|█▌        | 6/38 [00:25<01:21,  2.56s/it]\n",
      "Loading checkpoint shards:  18%|█▊        | 7/38 [00:25<00:58,  1.90s/it]\n",
      "Loading checkpoint shards:  21%|██        | 8/38 [00:26<00:43,  1.46s/it]\n",
      "Loading checkpoint shards:  24%|██▎       | 9/38 [00:26<00:33,  1.14s/it]\n",
      "Loading checkpoint shards:  26%|██▋       | 10/38 [00:27<00:26,  1.05it/s]\n",
      "Loading checkpoint shards:  29%|██▉       | 11/38 [00:27<00:22,  1.22it/s]\n",
      "Loading checkpoint shards:  32%|███▏      | 12/38 [00:28<00:19,  1.36it/s]\n",
      "Loading checkpoint shards:  34%|███▍      | 13/38 [00:28<00:16,  1.49it/s]\n",
      "Loading checkpoint shards:  37%|███▋      | 14/38 [00:29<00:16,  1.45it/s]\n",
      "Loading checkpoint shards:  39%|███▉      | 15/38 [00:30<00:20,  1.15it/s]\n",
      "Loading checkpoint shards:  42%|████▏     | 16/38 [00:31<00:17,  1.28it/s]\n",
      "Loading checkpoint shards:  45%|████▍     | 17/38 [00:32<00:15,  1.38it/s]\n",
      "Loading checkpoint shards:  47%|████▋     | 18/38 [00:32<00:13,  1.45it/s]\n",
      "Loading checkpoint shards:  50%|█████     | 19/38 [00:33<00:12,  1.54it/s]\n",
      "Loading checkpoint shards:  53%|█████▎    | 20/38 [00:33<00:11,  1.61it/s]\n",
      "Loading checkpoint shards:  55%|█████▌    | 21/38 [00:34<00:10,  1.65it/s]\n",
      "Loading checkpoint shards:  58%|█████▊    | 22/38 [00:34<00:09,  1.65it/s]\n",
      "Loading checkpoint shards:  61%|██████    | 23/38 [00:35<00:08,  1.67it/s]\n",
      "Loading checkpoint shards:  63%|██████▎   | 24/38 [00:36<00:08,  1.68it/s]\n",
      "Loading checkpoint shards:  66%|██████▌   | 25/38 [00:36<00:07,  1.70it/s]\n",
      "Loading checkpoint shards:  68%|██████▊   | 26/38 [00:37<00:07,  1.71it/s]\n",
      "Loading checkpoint shards:  71%|███████   | 27/38 [00:37<00:06,  1.63it/s]\n",
      "Loading checkpoint shards:  74%|███████▎  | 28/38 [00:38<00:05,  1.73it/s]\n",
      "Loading checkpoint shards:  76%|███████▋  | 29/38 [00:38<00:04,  1.84it/s]\n",
      "Loading checkpoint shards:  79%|███████▉  | 30/38 [00:39<00:04,  1.87it/s]\n",
      "Loading checkpoint shards:  82%|████████▏ | 31/38 [00:39<00:03,  1.90it/s]\n",
      "Loading checkpoint shards:  84%|████████▍ | 32/38 [00:40<00:03,  1.94it/s]\n",
      "Loading checkpoint shards:  87%|████████▋ | 33/38 [00:40<00:02,  1.99it/s]\n",
      "Loading checkpoint shards:  89%|████████▉ | 34/38 [00:41<00:01,  2.03it/s]\n",
      "Loading checkpoint shards:  92%|█████████▏| 35/38 [00:41<00:01,  2.05it/s]\n",
      "Loading checkpoint shards:  95%|█████████▍| 36/38 [00:42<00:00,  2.07it/s]\n",
      "Loading checkpoint shards:  97%|█████████▋| 37/38 [00:42<00:00,  2.09it/s]\n",
      "Loading checkpoint shards: 100%|██████████| 38/38 [00:43<00:00,  2.24it/s]\n",
      "Loading checkpoint shards: 100%|██████████| 38/38 [00:43<00:00,  1.14s/it]\n",
      "2024-03-17 10:36:20 | ERROR | stderr | \n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "2024-03-17 10:36:21 | INFO | model_worker | Register to controller\n"
     ]
    }
   ],
   "source": [
    "app = create_controller_app(\n",
    "    dispatch_method=FSCHAT_CONTROLLER.get(\"dispatch_method\"),\n",
    "    log_level=\"INFO\",\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "ConnectionError",
     "evalue": "HTTPConnectionPool(host='127.0.0.1', port=30001): Max retries exceeded with url: /register_worker (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f3f35dfb110>: Failed to establish a new connection: [Errno 111] Connection refused'))",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mConnectionRefusedError\u001b[0m                    Traceback (most recent call last)",
      "File \u001b[0;32m/home/liyaze/anaconda3/envs/longchain/lib/python3.11/site-packages/urllib3/connection.py:198\u001b[0m, in \u001b[0;36mHTTPConnection._new_conn\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    197\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 198\u001b[0m     sock \u001b[38;5;241m=\u001b[39m \u001b[43mconnection\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate_connection\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    199\u001b[0m \u001b[43m        \u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_dns_host\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mport\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    200\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    201\u001b[0m \u001b[43m        \u001b[49m\u001b[43msource_address\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msource_address\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    202\u001b[0m \u001b[43m        \u001b[49m\u001b[43msocket_options\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msocket_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    203\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    204\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m socket\u001b[38;5;241m.\u001b[39mgaierror \u001b[38;5;28;01mas\u001b[39;00m e:\n",
      "File \u001b[0;32m/home/liyaze/anaconda3/envs/longchain/lib/python3.11/site-packages/urllib3/util/connection.py:85\u001b[0m, in \u001b[0;36mcreate_connection\u001b[0;34m(address, timeout, source_address, socket_options)\u001b[0m\n\u001b[1;32m     84\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 85\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m err\n\u001b[1;32m     86\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m     87\u001b[0m     \u001b[38;5;66;03m# Break explicitly a reference cycle\u001b[39;00m\n",
      "File \u001b[0;32m/home/liyaze/anaconda3/envs/longchain/lib/python3.11/site-packages/urllib3/util/connection.py:73\u001b[0m, in \u001b[0;36mcreate_connection\u001b[0;34m(address, timeout, source_address, socket_options)\u001b[0m\n\u001b[1;32m     72\u001b[0m     sock\u001b[38;5;241m.\u001b[39mbind(source_address)\n\u001b[0;32m---> 73\u001b[0m \u001b[43msock\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43msa\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     74\u001b[0m \u001b[38;5;66;03m# Break explicitly a reference cycle\u001b[39;00m\n",
      "\u001b[0;31mConnectionRefusedError\u001b[0m: [Errno 111] Connection refused",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[0;31mNewConnectionError\u001b[0m                        Traceback (most recent call last)",
      "File \u001b[0;32m/home/liyaze/anaconda3/envs/longchain/lib/python3.11/site-packages/urllib3/connectionpool.py:793\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[0m\n\u001b[1;32m    792\u001b[0m \u001b[38;5;66;03m# Make the request on the HTTPConnection object\u001b[39;00m\n\u001b[0;32m--> 793\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_make_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    794\u001b[0m \u001b[43m    \u001b[49m\u001b[43mconn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    795\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    796\u001b[0m \u001b[43m    \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    797\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout_obj\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    798\u001b[0m \u001b[43m    \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    799\u001b[0m \u001b[43m    \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    800\u001b[0m \u001b[43m    \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    801\u001b[0m \u001b[43m    \u001b[49m\u001b[43mretries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mretries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    802\u001b[0m \u001b[43m    \u001b[49m\u001b[43mresponse_conn\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresponse_conn\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    803\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    804\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    805\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mresponse_kw\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    806\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    808\u001b[0m \u001b[38;5;66;03m# Everything went great!\u001b[39;00m\n",
      "File \u001b[0;32m/home/liyaze/anaconda3/envs/longchain/lib/python3.11/site-packages/urllib3/connectionpool.py:496\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[0;34m(self, conn, method, url, body, headers, retries, timeout, chunked, response_conn, preload_content, decode_content, enforce_content_length)\u001b[0m\n\u001b[1;32m    495\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 496\u001b[0m     \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    497\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    498\u001b[0m \u001b[43m        \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    499\u001b[0m \u001b[43m        \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    500\u001b[0m \u001b[43m        \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    501\u001b[0m \u001b[43m        \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    502\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpreload_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    503\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdecode_content\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    504\u001b[0m \u001b[43m        \u001b[49m\u001b[43menforce_content_length\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43menforce_content_length\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    505\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    507\u001b[0m \u001b[38;5;66;03m# We are swallowing BrokenPipeError (errno.EPIPE) since the server is\u001b[39;00m\n\u001b[1;32m    508\u001b[0m \u001b[38;5;66;03m# legitimately able to close the connection after sending a valid response.\u001b[39;00m\n\u001b[1;32m    509\u001b[0m \u001b[38;5;66;03m# With this behaviour, the received response is still readable.\u001b[39;00m\n",
      "File \u001b[0;32m/home/liyaze/anaconda3/envs/longchain/lib/python3.11/site-packages/urllib3/connection.py:400\u001b[0m, in \u001b[0;36mHTTPConnection.request\u001b[0;34m(self, method, url, body, headers, chunked, preload_content, decode_content, enforce_content_length)\u001b[0m\n\u001b[1;32m    399\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mputheader(header, value)\n\u001b[0;32m--> 400\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mendheaders\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    402\u001b[0m \u001b[38;5;66;03m# If we're given a body we start sending that in chunks.\u001b[39;00m\n",
      "File \u001b[0;32m/home/liyaze/anaconda3/envs/longchain/lib/python3.11/http/client.py:1293\u001b[0m, in \u001b[0;36mHTTPConnection.endheaders\u001b[0;34m(self, message_body, encode_chunked)\u001b[0m\n\u001b[1;32m   1292\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m CannotSendHeader()\n\u001b[0;32m-> 1293\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_send_output\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmessage_body\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencode_chunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencode_chunked\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/home/liyaze/anaconda3/envs/longchain/lib/python3.11/http/client.py:1052\u001b[0m, in \u001b[0;36mHTTPConnection._send_output\u001b[0;34m(self, message_body, encode_chunked)\u001b[0m\n\u001b[1;32m   1051\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_buffer[:]\n\u001b[0;32m-> 1052\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmsg\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1054\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m message_body \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m   1055\u001b[0m \n\u001b[1;32m   1056\u001b[0m     \u001b[38;5;66;03m# create a consistent interface to message_body\u001b[39;00m\n",
      "File \u001b[0;32m/home/liyaze/anaconda3/envs/longchain/lib/python3.11/http/client.py:990\u001b[0m, in \u001b[0;36mHTTPConnection.send\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m    989\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mauto_open:\n\u001b[0;32m--> 990\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mconnect\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    991\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n",
      "File \u001b[0;32m/home/liyaze/anaconda3/envs/longchain/lib/python3.11/site-packages/urllib3/connection.py:238\u001b[0m, in \u001b[0;36mHTTPConnection.connect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    237\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mconnect\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 238\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msock \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_new_conn\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    239\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_tunnel_host:\n\u001b[1;32m    240\u001b[0m         \u001b[38;5;66;03m# If we're tunneling it means we're connected to our proxy.\u001b[39;00m\n",
      "File \u001b[0;32m/home/liyaze/anaconda3/envs/longchain/lib/python3.11/site-packages/urllib3/connection.py:213\u001b[0m, in \u001b[0;36mHTTPConnection._new_conn\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    212\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m--> 213\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m NewConnectionError(\n\u001b[1;32m    214\u001b[0m         \u001b[38;5;28mself\u001b[39m, \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mFailed to establish a new connection: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    215\u001b[0m     ) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01me\u001b[39;00m\n\u001b[1;32m    217\u001b[0m \u001b[38;5;66;03m# Audit hooks are only available in Python 3.8+\u001b[39;00m\n",
      "\u001b[0;31mNewConnectionError\u001b[0m: <urllib3.connection.HTTPConnection object at 0x7f3f35dfb110>: Failed to establish a new connection: [Errno 111] Connection refused",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[0;31mMaxRetryError\u001b[0m                             Traceback (most recent call last)",
      "File \u001b[0;32m/home/liyaze/anaconda3/envs/longchain/lib/python3.11/site-packages/requests/adapters.py:486\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    485\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 486\u001b[0m     resp \u001b[38;5;241m=\u001b[39m \u001b[43mconn\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43murlopen\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    487\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    488\u001b[0m \u001b[43m        \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    489\u001b[0m \u001b[43m        \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbody\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    490\u001b[0m \u001b[43m        \u001b[49m\u001b[43mheaders\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mheaders\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    491\u001b[0m \u001b[43m        \u001b[49m\u001b[43mredirect\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    492\u001b[0m \u001b[43m        \u001b[49m\u001b[43massert_same_host\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    493\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpreload_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    494\u001b[0m \u001b[43m        \u001b[49m\u001b[43mdecode_content\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m    495\u001b[0m \u001b[43m        \u001b[49m\u001b[43mretries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmax_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    496\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtimeout\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtimeout\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    497\u001b[0m \u001b[43m        \u001b[49m\u001b[43mchunked\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunked\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    498\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    500\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (ProtocolError, \u001b[38;5;167;01mOSError\u001b[39;00m) \u001b[38;5;28;01mas\u001b[39;00m err:\n",
      "File \u001b[0;32m/home/liyaze/anaconda3/envs/longchain/lib/python3.11/site-packages/urllib3/connectionpool.py:847\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, preload_content, decode_content, **response_kw)\u001b[0m\n\u001b[1;32m    845\u001b[0m     new_e \u001b[38;5;241m=\u001b[39m ProtocolError(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mConnection aborted.\u001b[39m\u001b[38;5;124m\"\u001b[39m, new_e)\n\u001b[0;32m--> 847\u001b[0m retries \u001b[38;5;241m=\u001b[39m \u001b[43mretries\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mincrement\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    848\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43merror\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mnew_e\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_pool\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m_stacktrace\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msys\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexc_info\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m2\u001b[39;49m\u001b[43m]\u001b[49m\n\u001b[1;32m    849\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    850\u001b[0m retries\u001b[38;5;241m.\u001b[39msleep()\n",
      "File \u001b[0;32m/home/liyaze/anaconda3/envs/longchain/lib/python3.11/site-packages/urllib3/util/retry.py:515\u001b[0m, in \u001b[0;36mRetry.increment\u001b[0;34m(self, method, url, response, error, _pool, _stacktrace)\u001b[0m\n\u001b[1;32m    514\u001b[0m     reason \u001b[38;5;241m=\u001b[39m error \u001b[38;5;129;01mor\u001b[39;00m ResponseError(cause)\n\u001b[0;32m--> 515\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m MaxRetryError(_pool, url, reason) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mreason\u001b[39;00m  \u001b[38;5;66;03m# type: ignore[arg-type]\u001b[39;00m\n\u001b[1;32m    517\u001b[0m log\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mIncremented Retry for (url=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m): \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m\"\u001b[39m, url, new_retry)\n",
      "\u001b[0;31mMaxRetryError\u001b[0m: HTTPConnectionPool(host='127.0.0.1', port=30001): Max retries exceeded with url: /register_worker (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f3f35dfb110>: Failed to establish a new connection: [Errno 111] Connection refused'))",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mConnectionError\u001b[0m                           Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[7], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m worker \u001b[38;5;241m=\u001b[39m \u001b[43mModelWorker\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m      2\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcontroller_addr\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcontroller_address\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m      3\u001b[0m \u001b[43m    \u001b[49m\u001b[43mworker_addr\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mworker_address\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m      4\u001b[0m \u001b[43m    \u001b[49m\u001b[43mworker_id\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mworker_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m      5\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmodel_path\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmodel_path\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m      6\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmodel_names\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmodel_names\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m      7\u001b[0m \u001b[43m    \u001b[49m\u001b[43mlimit_worker_concurrency\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlimit_worker_concurrency\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m      8\u001b[0m \u001b[43m    \u001b[49m\u001b[43mno_register\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mno_register\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m      9\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdevice\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdevice\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     10\u001b[0m \u001b[43m    \u001b[49m\u001b[43mnum_gpus\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mnum_gpus\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     11\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmax_gpu_memory\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmax_gpu_memory\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     12\u001b[0m \u001b[43m    \u001b[49m\u001b[43mload_8bit\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mload_8bit\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     13\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcpu_offloading\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcpu_offloading\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     14\u001b[0m \u001b[43m    \u001b[49m\u001b[43mgptq_config\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mgptq_config\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     15\u001b[0m \u001b[43m    \u001b[49m\u001b[43mawq_config\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mawq_config\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     16\u001b[0m \u001b[43m    \u001b[49m\u001b[43mstream_interval\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstream_interval\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     17\u001b[0m \u001b[43m    \u001b[49m\u001b[43mconv_template\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconv_template\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     18\u001b[0m \u001b[43m    \u001b[49m\u001b[43membed_in_truncate\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43membed_in_truncate\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     19\u001b[0m \u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/home/liyaze/anaconda3/envs/longchain/lib/python3.11/site-packages/fastchat/serve/model_worker.py:104\u001b[0m, in \u001b[0;36mModelWorker.__init__\u001b[0;34m(self, controller_addr, worker_addr, worker_id, model_path, model_names, limit_worker_concurrency, no_register, device, num_gpus, max_gpu_memory, revision, dtype, load_8bit, cpu_offloading, gptq_config, awq_config, exllama_config, xft_config, stream_interval, conv_template, embed_in_truncate, seed, debug, **kwargs)\u001b[0m\n\u001b[1;32m    101\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mseed \u001b[38;5;241m=\u001b[39m seed\n\u001b[1;32m    103\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m no_register:\n\u001b[0;32m--> 104\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minit_heart_beat\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/home/liyaze/anaconda3/envs/longchain/lib/python3.11/site-packages/fastchat/serve/base_model_worker.py:79\u001b[0m, in \u001b[0;36mBaseModelWorker.init_heart_beat\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     78\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21minit_heart_beat\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m---> 79\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mregister_to_controller\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     80\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mheart_beat_thread \u001b[38;5;241m=\u001b[39m threading\u001b[38;5;241m.\u001b[39mThread(\n\u001b[1;32m     81\u001b[0m         target\u001b[38;5;241m=\u001b[39mheart_beat_worker,\n\u001b[1;32m     82\u001b[0m         args\u001b[38;5;241m=\u001b[39m(\u001b[38;5;28mself\u001b[39m,),\n\u001b[1;32m     83\u001b[0m         daemon\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m     84\u001b[0m     )\n\u001b[1;32m     85\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mheart_beat_thread\u001b[38;5;241m.\u001b[39mstart()\n",
      "File \u001b[0;32m/home/liyaze/anaconda3/envs/longchain/lib/python3.11/site-packages/fastchat/serve/base_model_worker.py:96\u001b[0m, in \u001b[0;36mBaseModelWorker.register_to_controller\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     90\u001b[0m url \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcontroller_addr \u001b[38;5;241m+\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/register_worker\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     91\u001b[0m data \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m     92\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mworker_name\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mworker_addr,\n\u001b[1;32m     93\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcheck_heart_beat\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[1;32m     94\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mworker_status\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_status(),\n\u001b[1;32m     95\u001b[0m }\n\u001b[0;32m---> 96\u001b[0m r \u001b[38;5;241m=\u001b[39m \u001b[43mrequests\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpost\u001b[49m\u001b[43m(\u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mjson\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     97\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m r\u001b[38;5;241m.\u001b[39mstatus_code \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m200\u001b[39m\n",
      "File \u001b[0;32m/home/liyaze/anaconda3/envs/longchain/lib/python3.11/site-packages/requests/api.py:115\u001b[0m, in \u001b[0;36mpost\u001b[0;34m(url, data, json, **kwargs)\u001b[0m\n\u001b[1;32m    103\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mpost\u001b[39m(url, data\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, json\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m    104\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124mr\u001b[39m\u001b[38;5;124;03m\"\"\"Sends a POST request.\u001b[39;00m\n\u001b[1;32m    105\u001b[0m \n\u001b[1;32m    106\u001b[0m \u001b[38;5;124;03m    :param url: URL for the new :class:`Request` object.\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    112\u001b[0m \u001b[38;5;124;03m    :rtype: requests.Response\u001b[39;00m\n\u001b[1;32m    113\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 115\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mpost\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdata\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mjson\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mjson\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/home/liyaze/anaconda3/envs/longchain/lib/python3.11/site-packages/requests/api.py:59\u001b[0m, in \u001b[0;36mrequest\u001b[0;34m(method, url, **kwargs)\u001b[0m\n\u001b[1;32m     55\u001b[0m \u001b[38;5;66;03m# By using the 'with' statement we are sure the session is closed, thus we\u001b[39;00m\n\u001b[1;32m     56\u001b[0m \u001b[38;5;66;03m# avoid leaving sockets open which can trigger a ResourceWarning in some\u001b[39;00m\n\u001b[1;32m     57\u001b[0m \u001b[38;5;66;03m# cases, and look like a memory leak in others.\u001b[39;00m\n\u001b[1;32m     58\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m sessions\u001b[38;5;241m.\u001b[39mSession() \u001b[38;5;28;01mas\u001b[39;00m session:\n\u001b[0;32m---> 59\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43msession\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m(\u001b[49m\u001b[43mmethod\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmethod\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43murl\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43murl\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/home/liyaze/anaconda3/envs/longchain/lib/python3.11/site-packages/requests/sessions.py:589\u001b[0m, in \u001b[0;36mSession.request\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m    584\u001b[0m send_kwargs \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m    585\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtimeout\u001b[39m\u001b[38;5;124m\"\u001b[39m: timeout,\n\u001b[1;32m    586\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mallow_redirects\u001b[39m\u001b[38;5;124m\"\u001b[39m: allow_redirects,\n\u001b[1;32m    587\u001b[0m }\n\u001b[1;32m    588\u001b[0m send_kwargs\u001b[38;5;241m.\u001b[39mupdate(settings)\n\u001b[0;32m--> 589\u001b[0m resp \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprep\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43msend_kwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    591\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m resp\n",
      "File \u001b[0;32m/home/liyaze/anaconda3/envs/longchain/lib/python3.11/site-packages/requests/sessions.py:703\u001b[0m, in \u001b[0;36mSession.send\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m    700\u001b[0m start \u001b[38;5;241m=\u001b[39m preferred_clock()\n\u001b[1;32m    702\u001b[0m \u001b[38;5;66;03m# Send the request\u001b[39;00m\n\u001b[0;32m--> 703\u001b[0m r \u001b[38;5;241m=\u001b[39m \u001b[43madapter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msend\u001b[49m\u001b[43m(\u001b[49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    705\u001b[0m \u001b[38;5;66;03m# Total elapsed time of the request (approximately)\u001b[39;00m\n\u001b[1;32m    706\u001b[0m elapsed \u001b[38;5;241m=\u001b[39m preferred_clock() \u001b[38;5;241m-\u001b[39m start\n",
      "File \u001b[0;32m/home/liyaze/anaconda3/envs/longchain/lib/python3.11/site-packages/requests/adapters.py:519\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m    515\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(e\u001b[38;5;241m.\u001b[39mreason, _SSLError):\n\u001b[1;32m    516\u001b[0m         \u001b[38;5;66;03m# This branch is for urllib3 v1.22 and later.\u001b[39;00m\n\u001b[1;32m    517\u001b[0m         \u001b[38;5;28;01mraise\u001b[39;00m SSLError(e, request\u001b[38;5;241m=\u001b[39mrequest)\n\u001b[0;32m--> 519\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(e, request\u001b[38;5;241m=\u001b[39mrequest)\n\u001b[1;32m    521\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m ClosedPoolError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    522\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mConnectionError\u001b[39;00m(e, request\u001b[38;5;241m=\u001b[39mrequest)\n",
      "\u001b[0;31mConnectionError\u001b[0m: HTTPConnectionPool(host='127.0.0.1', port=30001): Max retries exceeded with url: /register_worker (Caused by NewConnectionError('<urllib3.connection.HTTPConnection object at 0x7f3f35dfb110>: Failed to establish a new connection: [Errno 111] Connection refused'))"
     ]
    }
   ],
   "source": [
    "worker = ModelWorker(\n",
    "    controller_addr=controller_address,\n",
    "    worker_addr=worker_address,\n",
    "    worker_id=worker_id,\n",
    "    model_path=model_path,\n",
    "    model_names=model_names,\n",
    "    limit_worker_concurrency=limit_worker_concurrency,\n",
    "    no_register=no_register,\n",
    "    device=device,\n",
    "    num_gpus=num_gpus,\n",
    "    max_gpu_memory=max_gpu_memory,\n",
    "    load_8bit=load_8bit,\n",
    "    cpu_offloading=cpu_offloading,\n",
    "    gptq_config=gptq_config,\n",
    "    awq_config=awq_config,\n",
    "    stream_interval=stream_interval,\n",
    "    conv_template=conv_template,\n",
    "    embed_in_truncate=embed_in_truncate,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "(choose from 'cpu', 'cuda', 'mps', 'xpu', 'npu')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "longchain",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
